#!/bin/bash

#Submit this script with: sbatch thefilename

#SBATCH --time=720:00:00   # walltime
#SBATCH --ntasks-per-node=64
#SBATCH --nodes=1   # only 1, do not change. 
#SBATCH --mem=500G   # memory per node
#SBATCH -J "APP"   # job name


### Setup you app here
# for example, I want to use cp2k version 9.1, 
# it shoud be something like
# spack load cp2k@9.1
# use `spack find App_Name` to find out the exact strings for your application. 
# Install it yourself if it does not exist. 
spack load app@version-string

# set input string here
# for example, with out slurm 
# you can run cp2k.psmp by the following cmd, 
# mpirun -n 4 cp2k.psmp FullNameInputFile 1>cp2k.out 2>cp2k.err
# then your specficRunCMD will be: cp2k.psmp FullNameInputFile 1>cp2k.out 2>cp2k.err
# This is only an example. Please adapt it to suit your application.

export specficRunCMD="specific-cmd-to-run-you-app-including-full-input-file-name"

# LOAD MODULES, INSERT CODE, AND RUN YOUR PROGRAMS HERE
ulimit -d unlimited
ulimit -s unlimited
ulimit -t unlimited
ulimit -v unlimited

export SLURM_EXPORT_ENV=ALL


cd $SLURM_SUBMIT_DIR
username=`whoami`

### here, suppose /tmp is the I/O storage. 

APP_RUNDIR=/tmp/${username}.${SLURM_JOB_ID}

if [ ! -a $APP_RUNDIR ]; then
   echo "Scratch directory $APP_RUNDIR created."
   mkdir -p $APP_RUNDIR
fi

cp -r $SLURM_SUBMIT_DIR/* $APP_RUNDIR

cd $APP_RUNDIR


echo "Starting APP run at `hostname` on:" `date` >> ${SLURM_JOB_ID}.log

srun --mpi=pmi2 ${specficRunCMD}

echo "Finished APP run at `hostname `on:" `date` >> ${SLURM_JOB_ID}.log
cat $SLURM_JOB_NODELIST >> ${SLURM_JOB_ID}.log

cp -r $APP_RUNDIR/* $SLURM_SUBMIT_DIR
rm -fr $APP_RUNDIR

